package org.fujene.term;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.fujene.structure.Term;
import org.fujene.structure.TermList;

import net.java.sen.StringTagger;
import net.java.sen.Token;

/**
 * 
 * @author fujisaka_yusuke, accompanied with yasuda.
 *
 */
public class SenChopper implements Chopper {
	private static Map<String, Pattern> htmlTagNormalPattern = new HashMap<String, Pattern>();
	private static Map<String, Pattern> htmlTagClosePattern = new HashMap<String, Pattern>();
	private static Set<Character> invalidCharSet = new HashSet<Character>();

	private static String[] blackList = { "a", "b", "big", "blockquote", "br",
			"blink", "center", "em", "font", "hr", "i", "img", "marquee", "p",
			"s", "small", "strong", "strike", "u", "applet", "area", "bgsound",
			"body", "button", "caption", "comment", "dd", "div", "dl", "dt",
			"embed", "fieldset", "form", "frame", "frameset", "h1", "h2", "h3",
			"h4", "h5", "h6", "head", "html", "iframe", "input", "label",
			"legend", "li", "link", "map", "meta", "meta", "noframes",
			"object", "ol", "option", "param", "pre", "script", "select",
			"span", "style", "table", "tbody", "td", "textarea", "th", "title",
			"tr", "ul", "!--", "wbr", "nobr" };

	private static final char[] XML_INVALID_CHARS = { 0x0e, 0x00, 0x01, 0x02,
			0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0a, 0x0b, 0x0c, 0x0d, 0x0f,
			0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a,
			0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x7f };
	
	private static final char[] SPACE_CHARS = {'\u0009','\u000b','\u000c', '\u00a0', '\u0020', '	', '','	', '',' ','　'};
	private static Set<Character> spaceSet = new HashSet<Character>();
	private static List<Pattern> spacePattern = new ArrayList<Pattern>();

	static {
		for (String html : blackList) {
			Pattern patternNormalTag = Pattern.compile(html + "\\s+.*",
					(Pattern.CASE_INSENSITIVE) | (Pattern.UNICODE_CASE));
			Pattern patternCloseTag = Pattern.compile("/" + html,
					(Pattern.CASE_INSENSITIVE) | (Pattern.UNICODE_CASE));
			htmlTagNormalPattern.put(html, patternNormalTag);
			htmlTagClosePattern.put(html, patternCloseTag);
		}
		
		for(char invalidChar : XML_INVALID_CHARS) {
			invalidCharSet.add(invalidChar);
		}

		for(char space : SPACE_CHARS) {
			spaceSet.add(space);
			spacePattern.add(Pattern.compile(String.valueOf(space)));
		}
	}

	private String removeTags(String input) {
		if (input == null || input.length() == 0)
			return input;

		StringBuilder buf = new StringBuilder();
		char[] chars = input.toCharArray();
		boolean isTag = false;
		StringBuilder tagValue = null;

		for (int i = 0; i < chars.length; i++) {
			if (chars[i] == '<') {
				if (tagValue != null) {
					buf.append("<");
					buf.append(tagValue.toString());
				}
				tagValue = new StringBuilder();
				isTag = true;
			} else if (chars[i] == '>') {
				isTag = false;

				if (tagValue == null) {
					buf.append(">");
					continue;
				}

				boolean htmlTag = isTag(blackList, tagValue.toString());
				if (!htmlTag) {
					buf.append("<");
					buf.append(tagValue.toString());
					buf.append(">");
				}
				tagValue = null;
			} else if (!isTag) {
				buf.append(chars[i]);
			} else {
				if (tagValue == null) {
					continue;
				}

				tagValue.append(chars[i]);
			}
		}
		if (tagValue != null) {
			buf.append("<");
			buf.append(tagValue.toString());
		}
		return buf.toString();

	}

	private static boolean isTag(String[] htmlTagList, String tagValue) {
		for (String htmlTag : htmlTagList) {
			if (htmlTag.equals(tagValue.toLowerCase())) {
				return true;
			}
			if ((htmlTag + "/").equals(tagValue.toLowerCase())) {
				return true;
			}
			if (("/" + htmlTag).equals(tagValue.toLowerCase())) {
				return true;
			}

			Pattern patternNormalTag = htmlTagNormalPattern.get(htmlTag);
			Pattern patternCloseTag = htmlTagClosePattern.get(htmlTag);

			Matcher matcherNormalTag = patternNormalTag.matcher(tagValue);
			Matcher matcherCloseTag = patternCloseTag.matcher(tagValue);

			if (matcherNormalTag.matches() || matcherCloseTag.matches()) {
				return true;
			}
		}
		return false;
	}

	// 必ず末尾に入るの？
	private String removeFromiPhone(String input) {
		String iphoneFormat = "iPhoneからの投稿";

		if (input == null || input.length() == 0)
			return null;

		int pos = input.length() - iphoneFormat.length();
		if (pos > 0) {
			if (input.substring(pos).equals(iphoneFormat)) {
				input = input.substring(0, pos);
			}
		}

		return input;
	}

	/**
	 * 絵文字(私用領域文字)を除去します。<br>
	 * vodafone の WebCode
	 * 
	 * @param input
	 * @return 絵文字除去後のinput
	 */
	private String removeEmoji(String input) {
		if(input == null || input.length() == 0)
			return input;
		
		char[] chars = input.toCharArray();
		StringBuilder builder = new StringBuilder(input.length());

		// 絵文字(というか私用領域)の除去
		for (int i = 0; i < chars.length; i++) {
			if (!(chars[i] >= 0xE001 && chars[i] <= 0xF8FF)) {
				builder.append(chars[i]);
			}
		}

		// vodafone web文字を除去
		char[] webCode = { 0x001b, '$', 0x000f };
		int webCodeIndex = 0, firstIndex = 0, webCodeLen = 0;
		int bufLen = builder.length();
		for (int i = 0; i < bufLen; i++) {
			// webコードと一致しているかチェック
			if (webCode[webCodeIndex] == builder.charAt(i)) {
				if (webCodeIndex == 0) {
					firstIndex = i;
				} else if (webCodeIndex >= 2) {
					builder.delete(firstIndex, firstIndex + webCodeLen + 1);
					i -= webCodeLen + 1;
					bufLen = builder.length();

					firstIndex = 0;
					webCodeIndex = 0;
					webCodeLen = 0;
					continue;
				}
				webCodeIndex++;
				webCodeLen++;
			}
			// 制御コード間のカウント
			else if (webCodeIndex >= 2) {
				webCodeLen++;
			}
			// 一致していなければクリア
			else {
				firstIndex = 0;
				webCodeIndex = 0;
				webCodeLen = 0;
			}
		}

		return builder.toString();
	}

	private String removeControls(String input) {
		if(input == null || input.length() == 0)
			return input;

        StringBuilder sb = new StringBuilder();
        char[] chars = input.toCharArray();

   		for(char value : chars) {
   			if(invalidCharSet.contains(value))
   				continue;

   			sb.append(value);
        }

        return sb.toString();
	}
	
	private String shrinkWhiteSpace(String input) {
		if(input == null || input.length() == 0)
			return input;

        StringBuilder sb = new StringBuilder();
        char[] chars = input.toCharArray();
        char charOne = chars[0];
   		for(char value : chars) {
   			boolean spaceFlg = Character.isWhitespace(value);
   			boolean charOneSpaceFlg = Character.isWhitespace(charOne);

			if(!spaceFlg) {
   				charOne = value;
			} else if(charOneSpaceFlg && spaceFlg) {
   				charOne = value;
   				continue;
   			} else if(!charOneSpaceFlg && spaceFlg) {
   				charOne = value;
   			}
   			sb.append(value);
        }

        return sb.toString();
	}

	private String normalizeWhiteSpace(String input) {

		if(input == null || input.length() == 0)
			return input;

		for(Pattern pattern : spacePattern) {
			Matcher matcher = pattern.matcher(input);
			if (matcher.find())
				input = matcher.replaceAll("\u0020");
		}
		return input;
	}

	private String convert(String input) {
		if (input == null || input.length() == 0)
			return null;
		
		// Unicodeに変換

		// タグ除去
		input = removeTags(input);

		// iPhone投稿への対応
		input = removeFromiPhone(input);

		// 絵文字除去
		input = removeEmoji(input);
		
		// 制御文字除去
		input = removeControls(input);
		
		// 空白の統一
		input = normalizeWhiteSpace(input);
		
		// 連続空白の除去
		input = shrinkWhiteSpace(input);
		
		// 文字列の統一
		input = Escaper.convertSJISToMS932(input);
		
		// 半角カナ-> 全角カナ
		input = Optimizer.HanKanaToZen(input);
		
		// 全角アルファベット -> 半角
		input = Optimizer.ZenAlphaToHan(input);
		
		// アンエスケープ処理
		input = Escaper.unescapeEntityRef(input);
		
		// 半角記号 -> 全角記号
		input = Optimizer.HanSymbolToZen(input);
		
		// capital -> lower
		if(input != null)
			input = input.toLowerCase();
		
		return input;
	}
	
	private boolean isOnlyAscii(String str){
		char[] chars = str.toCharArray();
		
		for (int i = 0; i < chars.length; i++) {
			if(chars[i] < 0x0020 || chars[i] > 0x007E)
				return false;
		}
		
		return true;
	}

	@Override
	public TermList chop(String content) {
		TermList result = new TermList(content.length());
		String cleanedStr = null;
		
		if ((cleanedStr = convert(content)) != null) {
			try {
				StringTagger tagger = StringTagger.getInstance();
				String eachStr = null;
				
				Token[] token = tagger.analyze(cleanedStr);
				
				if(token != null){
					for (int i = 0; i < token.length; i++) {
						eachStr = token[i].toString();
						if(isOnlyAscii(eachStr)){
							// ASCII文字だけならそのまま追加
							result.add(new Term(eachStr));
						}else{
							// 1文字ずつ増やして追加
							for (int j = 1; j <= eachStr.length(); j++) {
								result.add(new Term(eachStr.substring(0, j)));
							}
						}
					}
				}
			} catch (IllegalArgumentException e) {
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

		return result;
	}
}
